package edu.trinity.cs.cjenkin1.webcrawler

import java.io.BufferedInputStream
import java.io.File
import java.io.FileOutputStream
import java.io.InputStream
import java.io.OutputStream
import java.net.URL

import scala.Array.canBuildFrom

import org.htmlcleaner.HtmlCleaner

class Crawler(url: String, nll: Int, lm: Crawler.CrawlStatus) {
import Crawler._
	val rootURL = tryNewURL(url)

	if (!rootURL.isEmpty) {
	  lm.linksFollowed_+=(List(rootURL.get))
	  lm.linksToFollow_+=(allHTMLLinks(url).filter
	      (_ isDefined).map(_ get).toList)
	  lm.imagesFound_+=(allImages(url).filter
	      (_ isDefined).map(_ get).toList)
		}
}

object Crawler {

  class CrawlStatus {
    import scala.collection.immutable.Set
    
    private var ltfSet = Set.empty[URL]
    private var ltfList = List[URL]()
    
    private var lfdSet = Set.empty[URL]
    private var lfdList = List[URL]()
    
    private var imfSet = Set.empty[URL]
    private var imfList = List[URL]()
    
    def linksToFollow = ltfList
    def linksFollowed = lfdList
    def imagesFound = imfList
    
    private def meta_+=(mset: Set[URL], mlist: List[URL])
    	(newLinks: List[URL]) = {
      val newElems = newLinks.toSet.diff(mset)
      (mset ++ newElems, mlist ++ (newElems.toList))
    }
    
    def linksToFollow_+=(newLinks: List[URL]) {
    	val (nset, nlist) = meta_+=(ltfSet,ltfList)(newLinks)
    	ltfSet = nset
    	ltfList = nlist
    }
    
    def linksFollowed_+=(newLinks: List[URL]) {
    	val (nset, nlist) = meta_+=(lfdSet,lfdList)(newLinks)
    	lfdSet = nset
    	lfdList = nlist
    }
    def imagesFound_+=(newLinks: List[URL]) {
    	val (nset, nlist) = meta_+=(imfSet,imfList)(newLinks)
    	imfSet = nset
    	imfList = nlist
    }    
    
    var numLinksLeft = 0;
  }

	def tryNewURL(url: String) = {
		try {
			Some(new URL(url) )
		}
		catch {
			case e:Exception => {
				println("Could not open URL: " + url)
				println(e + "\n")
				None
			}
		}
	}

	def crawl(url: String, numLinks: Int) = {
	  var stat = new CrawlStatus()
	  new Crawler(url,numLinks,stat)
	  stat
	}

  /**
   * Meta function to make all implementations easier to fix
   */
  def getTagAttributeRec(tag: String)(attr: String)(url: String) = {
    val cleaner = new HtmlCleaner()
    val node = cleaner.clean(new URL(url))

    val links = node.getElementsByName(tag, true)

    for (link <- links) yield {
      val linkText = link.getAttributeByName(attr)
      if (linkText == null || linkText.isEmpty) {
        None
      } else {
        val tryme = tryNewURL(linkText)
        if (tryme == None) {
          if (url.last == '/' || linkText.head == '/') {
            tryNewURL(url + linkText)
          } else
            tryNewURL(url + '/' + linkText)
        } else tryme
      }
    }
  }

  /**
   * Returns an array containing the URLs of all of the HTML links on the given page.
   */
	val allHTMLLinks = getTagAttributeRec("a")("href")_

  /**
   * Returns an array containing the URLs of all of the images on the given page.
   */
	val allImages = getTagAttributeRec("img")("src")_

  /**
   * Writes an {@code InputStream} to an {@code OutputStream} until EOF is reached.
   */
  private def write(in: InputStream, out: OutputStream, buf: Array[Byte], off: Int, len: Int) {
    var read = in.read(buf, off, len)
    while (read != -1) {
      out.write(buf, off, read)
      read = in.read(buf, off, len)
    }
  }

  /**
   * Downloads a file.
   */
  def download(url: URL, outFile: File, buf: Array[Byte], off: Int, len: Int) {
    val in = new BufferedInputStream(url.openStream());
    val out = new FileOutputStream(outFile)

    write(in, out, buf, off, len)

    in.close();
    out.close();
  }

  /**
   * Downloads a file to a temporary file and returns the temporary file.
   */
  def downloadTemp(url: URL, buf: Array[Byte], off: Int, len: Int) = {
    val outFile = File.createTempFile("image-crawler", null)
    download(url, outFile, buf, off, len)
    outFile
  }
}
